{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prerequisites\n",
    "Install the `opensourceai` sdk from `github` and specify your cluster information in `~/.opensourceai/clusters.yaml`. \n",
    "\n",
    "And for simplicity and security, we recommand user to setup necessary information in `.opensourceai/defaults.json` other than shown in the example notebook. (Refer to for [README](https://github.com/opensource-china/pai/blob/sdk-release-v0.4.00/contrib/python-sdk/README.md) more details.)\n",
    "\n",
    "_Please make sure you have set default values for ***cluster-alias***. This notebook will not set them explicitly for security and privacy issue_\n",
    "\n",
    "If not, use below commands to set them\n",
    "```bash\n",
    "opai set cluster-alias=<your/cluster/alias>\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from opensourceaisdk.command_line import Engine\n",
    "from opensourceaisdk.core import ClusterList, in_job_container\n",
    "from uuid import uuid4 as randstr\n",
    "\n",
    "clusters = Engine().process(['cluster', 'list'])\n",
    "default_values = Engine().process(['set'])\n",
    "print(default_values)\n",
    "\n",
    "cluster_alias = default_values[\"cluster-alias\"]\n",
    "assert cluster_alias in clusters, \"please specify cluster-alias and workspace\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submit jobs\n",
    "\n",
    "Now we submit jobs from \n",
    "- an existing version 1 job config file\n",
    "- an existing version 2 job config file\n",
    "- a hello-world command line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile mnist_v1.json\n",
    "{\n",
    "    \"jobName\": \"keras_tensorflow_backend_mnist\",\n",
    "    \"image\": \"opensourceai/pai.example.keras.tensorflow:stable\",\n",
    "    \"taskRoles\": [\n",
    "        {\n",
    "            \"name\": \"mnist\",\n",
    "            \"taskNumber\": 1,\n",
    "            \"cpuNumber\": 4,\n",
    "            \"memoryMB\": 8192,\n",
    "            \"gpuNumber\": 1,\n",
    "            \"command\": \"python mnist_cnn.py\"\n",
    "        }\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile mnist_v2.yaml\n",
    "protocolVersion: 2\n",
    "name: keras_tensorflow_mnist\n",
    "type: job\n",
    "version: 1.0\n",
    "contributor: opensourceai\n",
    "description: |\n",
    "  # Keras Tensorflow Backend MNIST Digit Recognition Examples\n",
    "  Trains a simple convnet on the MNIST dataset.\n",
    "  Gets to 99.25% test accuracy after 12 epochs\n",
    "  (there is still a lot of margin for parameter tuning).\n",
    "  16 seconds per epoch on a GRID K520 GPU.\n",
    "\n",
    "  Reference https://github.com/keras-team/keras/blob/master/examples/mnist_cnn.py\n",
    "\n",
    "prerequisites:\n",
    "  - protocolVersion: 2\n",
    "    name: keras_tensorflow_example\n",
    "    type: dockerimage\n",
    "    version: 1.0\n",
    "    contributor : opensourceai\n",
    "    description: |\n",
    "      This is an [example Keras with TensorFlow backend Docker image on opensourceai](https://github.com/opensource-china/pai/tree/master/examples/keras).\n",
    "    uri : opensourceai/pai.example.keras.tensorflow\n",
    "\n",
    "taskRoles:\n",
    "  train:\n",
    "    instances: 1\n",
    "    completion:\n",
    "      minSucceededInstances: 1\n",
    "    dockerImage: keras_tensorflow_example\n",
    "    resourcePerInstance:\n",
    "      cpu: 4\n",
    "      memoryMB: 8192\n",
    "      gpu: 1\n",
    "    commands:\n",
    "      - python mnist_cnn.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tests = [\"submit_v1\", \"submit_v2\", \"sub_oneliner\"]\n",
    "jobnames = {k: k + '_' + randstr().hex for k in tests}\n",
    "\n",
    "options = \"\"\n",
    "# options += \" --preview\"\n",
    "\n",
    "if not in_job_container():\n",
    "    jobs, cmds = [], []\n",
    "    \n",
    "    # submit v1\n",
    "    jobs.append(\"submit_v1_\" + randstr().hex)\n",
    "    cmds.append(f'opai job submit {options} --update jobName={jobs[-1]} mnist_v1.json')\n",
    "\n",
    "    # submit v2\n",
    "    jobs.append(\"submit_v2_\" + randstr().hex)\n",
    "    cmds.append(f'opai job submit {options} --update name={jobs[-1]} mnist_v2.yaml')\n",
    "    \n",
    "    # sub\n",
    "    jobs.append(\"sub_\" + randstr().hex)    \n",
    "    resource = '-i opensourceai/pai.example.keras.tensorflow --cpu 4 --memoryMB 8192 --gpu 1'\n",
    "    cmds.append(f'opai job sub {options} -j {jobs[-1]} {resource} python mnist_cnn.py')\n",
    "\n",
    "    # notebook\n",
    "    jobs.append(\"notebook_\" + randstr().hex)    \n",
    "    cmds.append(f'opai job notebook {options} -j {jobs[-1]} {resource} --python python3 --pip-installs keras 2-submit-job-from-local-notebook.ipynb')\n",
    "\n",
    "    for cmd in cmds:\n",
    "        print(cmd, \"\\n\")\n",
    "        ! {cmd}\n",
    "        print(\"\\n\")\n",
    "    \n",
    "    states = ClusterList().load().get_client(cluster_alias).wait(jobs)\n",
    "    failed_jobs = [t for i, t in enumerate(jobs) if states[i] != \"SUCCEEDED\"]\n",
    "    assert not failed_jobs, \"some of jobs fails %s\" % failed_jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
